#!/usr/bin/env python
#-*- coding:utf-8 -*-
import sys,os,re
from wwpy.db.DBI import DBI,dbc
from wwpy.util.Time import Time
from wwpy.io.Source import Source
from wwpy.util.Threads import ThreadPool

class MetaData(object):
    
    fields='meta_id,meta_type,meta_work,meta_acl,data_man,man_phone,man_mail,data_type,var_kws,data_path,data_name,data_fields,data_dims,data_mets,data_sep,data_fmt,time_dims,time_field'.split(',')
    comments='元数据ID|元数据类型|元数据业务|元数据权限控制|数据负责人|负责人手机号|负责人邮箱|数据类型(File,Table,Url)|变量关键词(${channel},${date},${hour},${year_month},${year_week},${year}等)|数据路径(可以使用*和var_kws中的变量)|数据名称(可以使用*和var_kws中的变量)|数据字段(多个用,分隔)|数据维度(多个用,分隔)|数据指标(多个用,分隔)|数据分隔符|数据格式|时间维度(Day,Hour,Week,Month,Year)|时间字段'.split('|')
    types='bigint(20)|varchar(30)|varchar(30)|varchar(30)|varchar(30)|bigint(20)|varchar(50)|varchar(30)|varchar(50)|varchar(250)|varchar(250)|varchar(250)|varchar(250)|varchar(250)|varchar(30)|varchar(250)|varchar(30)|varchar(50)'.split('|')
    table_fields=["%s %s AUTO_INCREMENT COMMENT '%s'" % (fields[0],types[0],comments[0])]
    table_fields.extend(["%s %s COMMENT '%s'" % (fields[idx],types[idx],comments[idx]) for idx in range(1,len(fields))])
    table_fields.append('primary key (meta_id)')
    field_comments=dict(zip(fields,comments))
    field_types=dict(zip(fields,types))
    table='mms_metadata'
    
    def __init__(self,meta_id=None,meta_type='Database',meta_work='58',meta_acl=None,
                 data_man='王伟',man_phone=13401145805,man_mail='wangwei02@58.com',
                 data_type='File',var_kws=None,data_path=None,data_name=None,
                 data_fields=None,data_dims=None,data_mets=None,data_sep='\\t',data_fmt=None,
                 time_dims='Date',time_field='stat_date'):
        self.meta_id=meta_id
        self.meta_type=meta_type #ETL or Database or Report
        self.meta_work=meta_work #58,招聘,黄页,精准,智能,店铺,二手,团购
        self.meta_acl=meta_acl
        self.data_man=data_man
        self.man_phone=man_phone
        self.man_mail=man_mail
        self.data_type=data_type #File,Table,Url
        self.var_kws=var_kws #${channel},${date},${hour},${year_month},${year_week},${year}
        self.data_path=data_path #可以使用*和var_kws中的变量
        self.data_name=data_name #可以使用*和var_kws中的变量
        self.data_fields=','.join(list(data_fields)) if isinstance(data_fields,(tuple,list,set)) else data_fields#多个用,分隔
        self.data_dims=','.join(list(data_dims)) if isinstance(data_dims,(tuple,list,set)) else data_dims #多个用,分隔
        self.data_mets=','.join(list(data_mets)) if isinstance(data_mets,(tuple,list,set)) else data_mets #多个用,分隔
        self.data_sep=data_sep
        self.data_fmt=data_fmt
        self.time_dims=','.join(list(time_dims)) if isinstance(time_dims,(tuple,list,set)) else time_dims #Date,Day,Hour,Week,Month,Year
        self.time_field=time_field #stat_date
        
    def __str__(self):
        return 'MetaData('+','.join((['='.join([field,repr(self.__dict__[field])]) for field in self.keys]))+')'
    
    @property
    def keys(self):
        return MetaData.fields
    
    @property
    def values(self):
        return [self.__dict__[field] for field in self.keys]
    
    @property
    def items(self):
        return zip(self.keys(),self.values)
    
    
class MetaDataRecord(object):
    fields='record_id,stat_date,meta_id,start_time,end_time,data_type,meta_data,data_size,data_lines'.split(',')
    comments='记录ID|统计日期|元数据ID|开始时间|结束时间|数据类型|元数据|数据大小|数据行数'.split('|')
    types='bigint(20)|date|bigint(20)|datetime|datetime|varchar(30)|varchar(300)|double|bigint(20)'.split('|')
    table_fields=["%s %s AUTO_INCREMENT COMMENT '%s'" % (fields[0],types[0],comments[0])]
    table_fields.extend([' '.join((fields[idx],types[idx],'comment',"'%s'" % comments[idx])) for idx in range(1,len(fields))])
    table_fields.append('primary key (record_id)')
    field_comments=dict(zip(fields,comments))
    field_types=dict(zip(fields,types))
    table='mms_metadata_record'
    
    def __init__(self,record_id=None,stat_date=None,meta_id=None,start_time=None,end_time=None,data_type='File',meta_data=None,data_size=0,data_lines=0):
        self.record_id=record_id
        self.stat_date=stat_date
        self.meta_id=meta_id
        self.start_time=start_time
        self.end_time=end_time
        self.data_type=data_type
        self.meta_data=meta_data
        self.data_size=data_size
        self.data_lines=data_lines
        
    def __str__(self):
        return 'MetaDataRecord('+','.join((['='.join([field,repr(self.__dict__[field])]) for field in self.keys]))+')'
    
    @property
    def keys(self):
        return MetaDataRecord.fields
    
    @property
    def values(self):
        return [self.__dict__[field] for field in self.keys]
    
    @property
    def items(self):
        return zip(self.keys(),self.values)
    
class MMS(object):
    
    def __init__(self,date=Time.yesterday(),hour=None):
        self.date=date
        self.hour=hour
        if not self.dbi.has_table(MetaData.table):
            self.create_metadata_table()
        if not self.dbi.has_table(MetaDataRecord.table):
            self.create_metadata_record_table()
            
    @property
    def dbi(self):
        return DBI(**dbc.db_data_ms)
    
    def create_metadata_table(self):
        self.dbi.create(MetaData.table,MetaData.table_fields)
        
    def insert_metadata(self,metadata):
        self.dbi.insert(MetaData.table,metadata.keys[1:],tuple(metadata.values[1:]))
        
    def load_metadata(self,metadata_file,sep='|'):
        self.dbi.load(metadata_file,MetaData.table,field_sep=sep)
        
    def create_metadata_record_table(self):
        self.dbi.create(MetaDataRecord.table,MetaDataRecord.table_fields)
    
    def insert_metadata_record(self,metadata_record):
        self.dbi.insert(MetaDataRecord.table,metadata_record.keys[1:],tuple(metadata_record.values[1:]))
        
    def load_metadata_record(self,metadata_record_file,sep='|'):
        self.dbi.load(metadata_record_file,MetaDataRecord.table,field_sep=sep)
        
    @property
    def var_kws_dict(self):
        return {
            '${date}':Time.date_format(self.date,''),
            '${year_month}':Time.date_year_month(self.date),
            '${year_week}':Time.date_year_week(self.date),
            '${year}':Time.date_year(self.date),
            '${hour}':map(lambda hour:Time.hour_format(hour),range(24)),
            '${channel}':'hy,dp,fc,es,zp,pw,cz,qc,jy,other'.split(','),
        }
    
    @property
    def date_dict(self):
        return dict(
            Date=Time.date_format(self.date,'-'),
            Year=Time.date_year(self.date),
            Month=Time.date_month(self.date),
            Day=Time.date_day(self.date),
            Week=Time.date_week(self.date),
            Hour=self.hour,
        )
    
    def get_metadatas(self):
        sql="SELECT * FROM %s " % MetaData.table
        vkd=self.var_kws_dict
        rows=self.dbi.query(sql)
        def _data_sep(data_type):
            if data_type=='File':
                return os.sep
            elif data_type=='Table':
                return '.'
            return '/'
        
        def _data_time(time_dim,time_field):
            ts=time_dim.split(',')
            d=self.date_dict
            return ' AND '.join([("%s(%s)=%s " % (t,time_field,repr(d.get(t))) if time_field!='stat_date' else "stat_date=%s " % (repr(d.get(t)))) for t in ts])
        
        def _data(data_type,data_path,data_name,time_dim,time_field):
            if data_path is None or data_name is None:
                return None
            ds=_data_sep(data_type)
            meta_data=ds.join((data_path,data_name))
            if data_type=='Table':
                real_data="SELECT COUNT(1) FROM %s WHERE 1=1 " % (meta_data)
                if time_field is not None:
                    real_data+='AND '+_data_time(time_dim,time_field)
                return meta_data,real_data
            return meta_data,meta_data
        metadatas=[]
        for row in rows:
            data_type=row['data_type']
            var_kws=row['var_kws']
            kws=[] if not var_kws else var_kws.split(',')
            data_path=row['data_path']
            data_name=row['data_name']
            time_dims=row['time_dims']
            time_field=row['time_field']
            meta_id=row['meta_id']
            lkws=[]
            for kw in kws:
                kwv=vkd.get(kw,None)
                if not isinstance(kwv,list):
                    data_path=data_path.replace(kw,kwv)
                    data_name=data_name.replace(kw,kwv)
                else:
                    lkws.append(kw)
            if len(lkws):
                dp,dn=data_path,data_name
                for kw in lkws:
                    kwv=vkd.get(kw,None)
                    for v in kwv:
                        data_path=dp.replace(kw,v)
                        data_name=dn.replace(kw,v)
                        meta_data,real_data=_data(data_type,data_path,data_name,time_dims,time_field)
                        metadata=dict(meta_id=meta_id,meta_data=meta_data,real_data=real_data,data_type=data_type)
                        metadatas.append(metadata)
            else:
                meta_data,real_data=_data(data_type,data_path,data_name,time_dims,time_field)
                metadata=dict(meta_id=meta_id,meta_data=meta_data,real_data=real_data,data_type=data_type)
                metadatas.append(metadata)
        return metadatas
    
    def check_metadata(self,metadata):
        mdr=MetaDataRecord()
        mdr.stat_date=self.date
        mdr.meta_id=metadata['meta_id']
        mdr.data_type=metadata['data_type']
        mdr.meta_data=metadata['meta_data']
        mdr.start_time=Time.now()
        s=Source(metadata['real_data'])
        if s.is_file or s.is_url:
            mdr.data_size=s.get_size()
        if s.is_sql:
            rows=[row for row in s.read()]
            mdr.data_lines=int(rows[0][0])
        else:
            mdr.data_lines=s.get_line_count()
        mdr.end_time=Time.now()
        self.insert_metadata_record(mdr)
        if mdr.data_lines==0:
            self.do_alert(mdr)
        return mdr
    
    def do_alert(self,metadata):
        from wwpy.util.Url import Url
        msg="<MMS Alert>No %s record in %s %s!" % (self.date,metadata.data_type,metadata.meta_data)
        Url.send(msg,self.mobiles)
        
    @property
    def mobiles(self):
        return '13401145805'
    
    def check_all(self):
        metadatas=self.get_metadatas()
        tp=ThreadPool(4)
        for metadata in metadatas:
            tp.put(self.check_metadata,metadata)
        tp.wait()
        records=[]
        for out in tp.outs():
            record=out[-1]
            records.append(record)
        return records
    
if __name__ == '__main__':
    mms=MMS()
    md=MetaData(meta_work='精准',var_kws='${date},${hour}',data_type='Url',data_path='http://10.5.12.167:50075/streamFile/dsap/rawdata/jzcpclog/${date}',data_name='jingzhun_clicks_${date}${hour}')
    mms.insert_metadata(md)
    for r in mms.check_all():
        print r
    